package 地区数据;

import Moudle.*;
import Util.HttpConnect;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.UpdateOptions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.ProtocolException;
import org.bson.Document;

import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.*;

public class Main {
    /*
     * 运行时 只需根据需要 更换time_code的值
     * 抓取一段时间后 网站会禁止你的访问 需等待 10~20分钟 再次运行
     * */


    /*
     * itemMap：包含所有已经抓取的信息
     * localFile：本地已存在的文件
     * */
    static String FENSHENG = "分省年度";
    static Map<String, Item> itemMap = new HashMap<>();
    static Set<String> localFile;

    static String db_code; //fsyd 分省月度
    static City[] cities;
    static int sleep_time = 1000;

    static MongoClient mongoClient = new MongoClient("192.168.1.173", 27017);
    static MongoDatabase mDatabase = mongoClient.getDatabase("国家统计局");
    static MongoCollection mCollection = mDatabase.getCollection(FENSHENG);

    public static void main(String[] args) throws InterruptedException, ProtocolException, CantVisitTheWePageException, IOException {
        START();
    }

    static void START() throws InterruptedException {
        try {
            db_code = "fsnd";
            cities = GetAllCity();
            localFile = getAllFile("D:\\测试临时文件\\国家统计局-"+FENSHENG, false);
            if (cities != null && cities.length > 0)
                System.out.println("获取城市信息成功");

            TraverseNode("zb");

        } catch (Exception e) {
            e.printStackTrace();

            if (e instanceof CantVisitTheWePageException) {
                System.out.println("---休眠二十分钟----");
                Thread.sleep(1200000);
                START();
            }
        }
    }

    static void TraverseNode(String id) throws CantVisitTheWePageException, NullPointerException, ProtocolException, InterruptedException, IOException {
        String url = "https://data.stats.gov.cn/easyquery.htm";

        Map<String, String> header_map = new HashMap<>();
        header_map.put("Content-Type", "application/x-www-form-urlencoded");

        Map<String, String> data_map = new HashMap<>();
        data_map.put("id", id);
        data_map.put("dbcode", db_code);
        data_map.put("wdcode", "zb");
        data_map.put("m", "getTree");

        Thread.sleep(sleep_time);
        System.out.println("POST id =" + id);
        String jsonStr = HttpConnect.HttpPost_OkHttp(url, header_map, data_map);

        if (jsonStr == null) {
            throw new NullPointerException(jsonStr);
        }

        if (!Can_VisitTheWebPage(jsonStr)) //无法访问网页一段事时间后 再次运行
            throw new CantVisitTheWePageException(jsonStr);


        Item[] items = ParseData.parseJsonStrToItem(jsonStr);
        for (Item item : items) {
            itemMap.put(item.id, item);
            if (item.isParent.equals("true")) {
                TraverseNode(item.id);
            } else {//叶节点
                if (!localFile.contains(item.name)) //已在本地 不再抓取
                    getData(item);
            }
        }
    }

    //所有数据都在叶节点
    static void getData(Item item) throws CantVisitTheWePageException, IOException, InterruptedException {
        //遍历每个城市
        try {
            List<Excel_地区数据> excel_list = new ArrayList<>();
            List<String> zb_list = new ArrayList<>();
            String excel_ZHU = "";
            for (City city : cities) {

                String time = "2020";
                String wds_source = "[{\"wdcode\":\"reg\",\"valuecode\":\"" + city.code + "\"}]";
                String dfwds_source = "[{\"wdcode\":\"zb\",\"valuecode\":\"" + item.id + "\"},{\"wdcode\":\"sj\",\"valuecode\":\"" + time + "\"}]";

                String wds = URLEncoder.encode(wds_source, "UTF-8");
                String dfwds = URLEncoder.encode(dfwds_source, "UTF-8");
                String url = "https://data.stats.gov.cn/easyquery.htm?m=QueryData&dbcode=" + db_code + "&rowcode=zb&colcode=sj&wds=" + wds + "&dfwds=" + dfwds;

                Thread.sleep(sleep_time);
                System.out.println("GET item.name = " + item.name + "   City_name = " + city.name);
                String jsonStr = HttpConnect.HttpGet_OkHttp(url, null);

                if (!Can_VisitTheWebPage(jsonStr)) //无法访问网页一段事时间后 再次运行
                    throw new CantVisitTheWePageException(item.name);
                String ZHU = ParseData.parseData(jsonStr, city, excel_list, zb_list);
                if (!excel_ZHU.contains(ZHU))
                    excel_ZHU = excel_ZHU + ZHU;
                //请求太快 网站会拒绝访问 在这停顿2秒
                Thread.sleep(sleep_time);
            }
            ToExcel.toExcel(excel_list, item, zb_list, FENSHENG);
            InsertToDb(item.name, excel_ZHU);
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
            System.out.println("-------------encode_err------------\n\n");
        } catch (IOException e) {
            e.printStackTrace();
        } catch (JSONException e) {
            e.printStackTrace();
            System.out.println("getData()  " + item.name + " 进行重试");
        }
    }

    public static Set<String> getAllFile(String directoryPath, boolean isAddDirectory) {
        //获取本地所有文件
        Set<String> list = new HashSet<>();
        File baseFile = new File(directoryPath);
        if (baseFile.isFile() || !baseFile.exists()) {
            return list;
        }
        File[] files = baseFile.listFiles();
        for (File file : files) {
            if (file.isDirectory()) {
                if (isAddDirectory) {
                    list.add(file.getAbsolutePath());
                }
                list.addAll(getAllFile(file.getAbsolutePath(), isAddDirectory));
            } else {
                String fileName = file.getName();
                list.add(fileName.substring(0, fileName.indexOf(".")));
            }
        }
        return list;
    }


    //获取所有 地区信息
    static City[] GetAllCity() throws CantVisitTheWePageException {
        String url = "https://data.stats.gov.cn/easyquery.htm?m=getOtherWds&dbcode=fsyd&rowcode=zb&colcode=sj&wds=%5B%5D&k1=1621480102727";
        String jsonStr = HttpConnect.HttpGet_OkHttp(url, null);

        if (!Can_VisitTheWebPage(jsonStr)) //无法访问网页一段事时间后 再次运行
            throw new CantVisitTheWePageException("GetAllCity()");
        return ParseData.parseJsonStrToCity(jsonStr);
    }


    static boolean Can_VisitTheWebPage(String str) {
        //网站有反爬虫 多次请求会拒绝访问
        if (str == null || str.contains("请开启JavaScript并刷新该页") || str.contains("307 Temporary Redirect")) {
            return false;
        }
        return true;
    }

    static void InsertToDb(String excel_name, String ZHU) {
        Document doc = new Document("_id", excel_name).append("注释", ZHU);
        BasicDBObject obj = new BasicDBObject("$set", doc);
        BasicDBObject query = new BasicDBObject().append("_id", excel_name);
        mCollection.updateOne(query, obj, (new UpdateOptions()).upsert(true));
        System.out.println(excel_name + "注释信息 写入数据库");
    }

    static void Append_new_collection(){
        /*这些表格 需要 95-20年数据*/

    }


}


